/*
 * Copyright (c) 2005 Topspin Communications.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#define _GNU_SOURCE
//#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netdb.h>
#include <malloc.h>
#include <getopt.h>
#include <arpa/inet.h>
#include <time.h>
#include <inttypes.h>

#include "pingpong.h"

#define MAX_CON_SIZE 1000000
#define MAX_COMMUNICATION_ITERS 1000000
#define max(a, b) (a) > (b) ? (a) : (b)
#define min(a, b) (a) < (b) ? (a) : (b)

//#include <ccan/minmax.h>
static int requiresize;
enum {
	PINGPONG_RECV_WRID = 1,
	PINGPONG_SEND_WRID = 2,
};

static int page_size;
static int use_odp;
static int use_ts;
static int validate_buf;
// struct timespec start, end;


struct pingpong_context {
	struct ibv_context	*context;
	struct ibv_comp_channel *channel;
	struct ibv_pd		*pd;
	struct ibv_mr		*mr;
	// union {
	// 	struct ibv_cq		*cq;
	// 	struct ibv_cq_ex	*cq_ex;
	// } cq_s;
    struct ibv_qp        **qp;    // 指向队列对数组的指针
    struct ibv_cq        **cq;    // 指向完成队列数组的指针
	// struct ibv_qp		*qp;
	char			*buf;
	int			 size;
	int			 send_flags;
	int			 rx_depth;
	int			 pending;
	struct ibv_port_attr     portinfo;
	uint64_t		 completion_timestamp_mask;
    int             num_servers;       // 服务器数量
    struct pingpong_dest    *rem_dest; // 改为指向多个服务器目的地信息的数组
};

// static struct ibv_cq *pp_cq(struct pingpong_context *ctx)
// {
// 	return use_ts ? ibv_cq_ex_to_cq(ctx->cq_s.cq_ex) :
// 		ctx->cq_s.cq;
// }

struct pingpong_dest {
	int lid;
	int qpn;
	int psn;
	union ibv_gid gid;
};

static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn,
			  enum ibv_mtu mtu, int sl,
			  struct pingpong_dest *dest, int sgid_idx, int z)
{   
    // printf("1234\n");
	struct ibv_qp_attr attr = {
		.qp_state		= IBV_QPS_RTR,
		.path_mtu		= mtu,
		.dest_qp_num		= dest->qpn,
		.rq_psn			= dest->psn,
		.max_dest_rd_atomic	= 1,
		.min_rnr_timer		= 12,
		.ah_attr		= {
			.is_global	= 0,
			.dlid		= dest->lid,
			.sl		= sl,
			.src_path_bits	= 0,
			.port_num	= port
		}
	};

	if (dest->gid.global.interface_id) {
		attr.ah_attr.is_global = 1;
		attr.ah_attr.grh.hop_limit = 1;
		attr.ah_attr.grh.dgid = dest->gid;
		attr.ah_attr.grh.sgid_index = sgid_idx;
	}
	if (ibv_modify_qp(ctx->qp[z], &attr,
			  IBV_QP_STATE              |
			  IBV_QP_AV                 |
			  IBV_QP_PATH_MTU           |
			  IBV_QP_DEST_QPN           |
			  IBV_QP_RQ_PSN             |
			  IBV_QP_MAX_DEST_RD_ATOMIC |
			  IBV_QP_MIN_RNR_TIMER)) {
		fprintf(stderr, "Failed to modify QP to RTR\n");
		return 1;
	}

	attr.qp_state	    = IBV_QPS_RTS;
	attr.timeout	    = 14;
	attr.retry_cnt	    = 7;
	attr.rnr_retry	    = 7;
	attr.sq_psn	    = my_psn;
	attr.max_rd_atomic  = 1;
	if (ibv_modify_qp(ctx->qp[z], &attr,
			  IBV_QP_STATE              |
			  IBV_QP_TIMEOUT            |
			  IBV_QP_RETRY_CNT          |
			  IBV_QP_RNR_RETRY          |
			  IBV_QP_SQ_PSN             |
			  IBV_QP_MAX_QP_RD_ATOMIC)) {
		fprintf(stderr, "Failed to modify QP to RTS\n");
		return 1;
	}

	return 0;
}

static struct pingpong_dest *pp_client_exch_dest(const char *servername, int port,
						 const struct pingpong_dest *my_dest, struct pingpong_dest *rem_dest)
{
	struct addrinfo *res, *t;
    // ipv4
    struct addrinfo hints = {
    .ai_family = AF_UNSPEC,
    .ai_socktype = SOCK_STREAM
    };
    // ipv6?
	// struct addrinfo hints;
	// memset(&hints, 0, sizeof hints);
	// hints.ai_family = AF_INET6;
	// hints.ai_socktype = SOCK_STREAM;
	char *service;
	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
	int n;
	int sockfd = -1;
	// struct pingpong_dest *rem_dest = NULL;
	char gid[33];
	if (asprintf(&service, "%d", port) < 0)
		return NULL;
	n = getaddrinfo(servername, service, &hints, &res);

	if (n != 0) {
		fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port);
		free(service);
		return NULL;
	}

	for (t = res; t; t = t->ai_next) {
		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
		if (sockfd >= 0) {
			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
				break;
			close(sockfd);
			sockfd = -1;
		}
	}

	freeaddrinfo(res);
	free(service);

	if (sockfd < 0) {
		fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
		return NULL;
	}

	gid_to_wire_gid(&my_dest->gid, gid);
	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
							my_dest->psn, gid);
	if (write(sockfd, msg, sizeof msg) != sizeof msg) {
		fprintf(stderr, "Couldn't send local address\n");
		goto out;
	}

	if (read(sockfd, msg, sizeof msg) != sizeof msg ||
	    write(sockfd, "done", sizeof "done") != sizeof "done") {
		perror("client read/write");
		fprintf(stderr, "Couldn't read/write remote address\n");
		goto out;
	}

	// rem_dest = malloc(sizeof *rem_dest);
	// if (!rem_dest)
	// 	goto out;

    sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
                              &rem_dest->psn, gid);
    wire_gid_to_gid(gid, &rem_dest->gid);
out:
	close(sockfd);
	return rem_dest;
}

// static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx,
// 						 int ib_port, enum ibv_mtu mtu,
// 						 int port, int sl,
// 						 const struct pingpong_dest *my_dest,
// 						 int sgid_idx)
// {
// 	struct addrinfo *res, *t;
// 	struct addrinfo hints = {
// 		.ai_flags    = AI_PASSIVE,
// 		.ai_family   = AF_UNSPEC,
// 		.ai_socktype = SOCK_STREAM
// 	};
// 	char *service;
// 	char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"];
// 	int n;
// 	int sockfd = -1, connfd;
// 	struct pingpong_dest *rem_dest = NULL;
// 	char gid[33];

// 	if (asprintf(&service, "%d", port) < 0)
// 		return NULL;

// 	n = getaddrinfo(NULL, service, &hints, &res);

// 	if (n < 0) {
// 		fprintf(stderr, "%s for port %d\n", gai_strerror(n), port);
// 		free(service);
// 		return NULL;
// 	}

// 	for (t = res; t; t = t->ai_next) {
// 		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
// 		if (sockfd >= 0) {
// 			n = 1;

// 			setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);

// 			if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
// 				break;
// 			close(sockfd);
// 			sockfd = -1;
// 		}
// 	}

// 	freeaddrinfo(res);
// 	free(service);

// 	if (sockfd < 0) {
// 		fprintf(stderr, "Couldn't listen to port %d\n", port);
// 		return NULL;
// 	}

// 	listen(sockfd, 1);
// 	connfd = accept(sockfd, NULL, NULL);
// 	close(sockfd);
// 	if (connfd < 0) {
// 		fprintf(stderr, "accept() failed\n");
// 		return NULL;
// 	}

// 	n = read(connfd, msg, sizeof msg);
// 	if (n != sizeof msg) {
// 		perror("server read");
// 		fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg);
// 		goto out;
// 	}

// 	rem_dest = malloc(sizeof *rem_dest);
// 	if (!rem_dest)
// 		goto out;

// 	sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn,
// 							&rem_dest->psn, gid);
// 	wire_gid_to_gid(gid, &rem_dest->gid);

// 	if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest,
// 								sgid_idx)) {
// 		fprintf(stderr, "Couldn't connect to remote QP\n");
// 		free(rem_dest);
// 		rem_dest = NULL;
// 		goto out;
// 	}


// 	gid_to_wire_gid(&my_dest->gid, gid);
// 	sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn,
// 							my_dest->psn, gid);
// 	if (write(connfd, msg, sizeof msg) != sizeof msg ||
// 	    read(connfd, msg, sizeof msg) != sizeof "done") {
// 		fprintf(stderr, "Couldn't send/recv local address\n");
// 		free(rem_dest);
// 		rem_dest = NULL;
// 		goto out;
// 	}


// out:
// 	close(connfd);
// 	return rem_dest;
// }
double tmp[MAX_COMMUNICATION_ITERS] = {0};
void merge_sort(double q[], int l, int r)
{
    if (l >= r) return;

    int mid = l + r >> 1;
    merge_sort(q, l, mid);
    merge_sort(q, mid + 1, r);

    int k = 0, i = l, j = mid + 1;
    while (i <= mid && j <= r)
        if (q[i] <= q[j]) tmp[k ++ ] = q[i ++ ];
        else tmp[k ++ ] = q[j ++ ];

    while (i <= mid) tmp[k ++ ] = q[i ++ ];
    while (j <= r) tmp[k ++ ] = q[j ++ ];

    for (i = l, j = 0; i <= r; i ++, j ++ ) q[i] = tmp[j];
}

static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
					    int rx_depth, int port, int server_count,
					    int use_event, int num_servers)
{
	struct pingpong_context *ctx;
	int access_flags = IBV_ACCESS_LOCAL_WRITE;

	ctx = calloc(1, sizeof *ctx);
	if (!ctx)
		return NULL;

	ctx->size       = size;
	ctx->send_flags = IBV_SEND_SIGNALED;
	ctx->rx_depth   = rx_depth;
    ctx->num_servers = num_servers;
	printf("Server Num: %d\n", num_servers);
	//分配QP和CQ的数组
    ctx->qp = calloc(num_servers, sizeof(*ctx->qp));
    ctx->cq = calloc(num_servers, sizeof(*ctx->cq));
    ctx->rem_dest = calloc(num_servers, sizeof(*ctx->rem_dest));
    if (!ctx->qp || !ctx->cq || !ctx->rem_dest) {
        fprintf(stderr, "Failed to allocate resources for the servers\n");
        goto clean_ctx; // 或相应的错误处理
    }

	int register_size = max(size, 64) * 16;
	ctx->buf = memalign(page_size, register_size);
	if (!ctx->buf) {
		fprintf(stderr, "Couldn't allocate work buf.\n");
		goto clean_ctx;
	}

	/* FIXME memset(ctx->buf, 0, size); */
	memset(ctx->buf, 0x7b, register_size);

	ctx->context = ibv_open_device(ib_dev);
	if (!ctx->context) {
		fprintf(stderr, "Couldn't get context for %s\n",
			ibv_get_device_name(ib_dev));
		goto clean_buffer;
	}

	if (use_event) {
		ctx->channel = ibv_create_comp_channel(ctx->context);
		if (!ctx->channel) {
			fprintf(stderr, "Couldn't create completion channel\n");
			goto clean_device;
		}
	} else
		ctx->channel = NULL;

	ctx->pd = ibv_alloc_pd(ctx->context);
	if (!ctx->pd) {
		fprintf(stderr, "Couldn't allocate PD\n");
		goto clean_comp_channel;
	}

	if (use_odp || use_ts) {
		const uint32_t rc_caps_mask = IBV_ODP_SUPPORT_SEND |
					      IBV_ODP_SUPPORT_RECV;
		struct ibv_device_attr_ex attrx;

		if (ibv_query_device_ex(ctx->context, NULL, &attrx)) {
			fprintf(stderr, "Couldn't query device for its features\n");
			goto clean_pd;
		}

		if (use_odp) {
			if (!(attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
			    (attrx.odp_caps.per_transport_caps.rc_odp_caps & rc_caps_mask) != rc_caps_mask) {
				fprintf(stderr, "The device isn't ODP capable or does not support RC send and receive with ODP\n");
				goto clean_pd;
			}
			access_flags |= IBV_ACCESS_ON_DEMAND;
		}

		if (use_ts) {
			if (!attrx.completion_timestamp_mask) {
				fprintf(stderr, "The device isn't completion timestamp capable\n");
				goto clean_pd;
			}
			ctx->completion_timestamp_mask = attrx.completion_timestamp_mask;
		}
	}
	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, register_size, access_flags);

	if (!ctx->mr) {
		fprintf(stderr, "Couldn't register MR\n");
		goto clean_pd;
	}


    for (int i = 0; i < num_servers; ++i) {

        ctx->cq[i] = ibv_create_cq(ctx->context, rx_depth + 1, NULL, ctx->channel, 0);
        if (!ctx->cq[i]) {
            fprintf(stderr, "Couldn't create CQ for server %d\n", i);
            goto clean_cq;
            
        }

        struct ibv_qp_attr attr;
        struct ibv_qp_init_attr init_attr = {
            .send_cq = ctx->cq[i],
            .recv_cq = ctx->cq[i],
            .cap = {
                .max_send_wr = rx_depth,//由一修改
                .max_recv_wr = rx_depth,
                .max_send_sge = 1, //由1改为了2
                .max_recv_sge = 1
                // .max_recv_sge = 1,
                // .max_inline_data = 0,
            },
            .qp_type = IBV_QPT_RC,
            // .sq_sig_all = 0,
        };
        ctx->qp[i] = ibv_create_qp(ctx->pd, &init_attr);
        if (!ctx->qp[i]) {
            fprintf(stderr, "Couldn't create QP for server %d\n", i);
            goto clean_qp;
        }

		ibv_query_qp(ctx->qp[i], &attr, IBV_QP_CAP, &init_attr);
		if (init_attr.cap.max_inline_data >= size) {
			ctx->send_flags |= IBV_SEND_INLINE;
		}
        {
            struct ibv_qp_attr attr = {
                .qp_state        = IBV_QPS_INIT,
                .pkey_index      = 0,
                .port_num        = port,
                .qp_access_flags = 0
            };

            if (ibv_modify_qp(ctx->qp[i], &attr,
                    IBV_QP_STATE              |
                    IBV_QP_PKEY_INDEX         |
                    IBV_QP_PORT               |
                    IBV_QP_ACCESS_FLAGS)) {
                fprintf(stderr, "Failed to modify QP to INIT\n");
                goto clean_qp;
            }
        }

        // 配置每个QP的状态，从RESET到INIT到RTR到RTS
        // struct ibv_qp_attr qp_attr;
        // memset(&qp_attr, 0, sizeof(qp_attr));
        // qp_attr.qp_state = IBV_QPS_INIT;
        // qp_attr.port_num = port;
        // qp_attr.pkey_index = 0;
        // qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE;

        // if (ibv_modify_qp(ctx->qp[i], &qp_attr,
        //                   IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
        //     fprintf(stderr, "Failed to modify QP to INIT state\n");
        //     goto clean_qp;
        // }
        
    }

	return ctx;

	clean_qp:
		for (int i = 0; i < num_servers; ++i) {
			if (ctx->qp[i]) ibv_destroy_qp(ctx->qp[i]);
		}

	clean_cq:
		for (int i = 0; i < num_servers; ++i) {
			if (ctx->cq[i]) ibv_destroy_cq(ctx->cq[i]);
		}

	clean_mr:
		ibv_dereg_mr(ctx->mr);

	clean_pd:
		ibv_dealloc_pd(ctx->pd);

	clean_comp_channel:
		if (ctx->channel)
			ibv_destroy_comp_channel(ctx->channel);

	clean_device:
		ibv_close_device(ctx->context);

	clean_buffer:
		free(ctx->buf);

	clean_ctx:
		free(ctx->cq);
		free(ctx->qp);
		free(ctx);

		return NULL;
}

static int pp_close_ctx(struct pingpong_context *ctx, int server_count)
{	
	int i = 0;
	for (i = 0; i < server_count; ++i) {
		if (ibv_destroy_qp(ctx->qp[i])) {
			fprintf(stderr, "Couldn't destroy QP\n");
			return 1;
		}

		
		if (ibv_destroy_cq(ctx->cq[i])) {
			fprintf(stderr, "Couldn't destroy CQ\n");
			return 1;
		}
	}
	if (ibv_dereg_mr(ctx->mr)) {
		fprintf(stderr, "Couldn't deregister MR\n");
		return 1;
	}

	if (ibv_dealloc_pd(ctx->pd)) {
		fprintf(stderr, "Couldn't deallocate PD\n");
		return 1;
	}

	if (ctx->channel) {
		if (ibv_destroy_comp_channel(ctx->channel)) {
			fprintf(stderr, "Couldn't destroy completion channel\n");
			return 1;
		}
	}

	if (ibv_close_device(ctx->context)) {
		fprintf(stderr, "Couldn't release context\n");
		return 1;
	}

	free(ctx->buf);
	free(ctx);

	return 0;
}

static int pp_post_recv_many(struct pingpong_context *ctx, int n, int z)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_recv_wr wr = {
		.wr_id	    = PINGPONG_RECV_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
	};
	struct ibv_recv_wr *bad_wr;
	int i;

	for (i = 0; i < n; ++i)
		if (ibv_post_recv(ctx->qp[z], &wr, &bad_wr))
			break;

	return i;
}

static int pp_post_recv(struct pingpong_context *ctx, int n)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_recv_wr wr = {
		.wr_id	    = PINGPONG_RECV_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
	};
	struct ibv_recv_wr *bad_wr;
	int i;

	for (i = 0; i < n; ++i)
		if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
			break;

	return i;
}

// static int reserve_pp_post_recv(struct pingpong_context *ctx, int n)
// {
// 	struct ibv_sge list = {
// 		.addr	= (uintptr_t) ctx->buf,
// 		.length = 32,
// 		.lkey	= ctx->mr->lkey
// 	};
// 	struct ibv_recv_wr wr = {
// 		.wr_id	    = PINGPONG_RECV_WRID,
// 		.sg_list    = &list,
// 		.num_sge    = 1,
// 	};
// 	struct ibv_recv_wr *bad_wr;
// 	int i;

// 	for (i = 0; i < n; ++i)
// 		if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
// 			break;

// 	return i;
// }

static int client_pp_post_recv(struct pingpong_context *ctx, int n)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_recv_wr wr = {
		.wr_id	    = PINGPONG_RECV_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
	};
	struct ibv_recv_wr *bad_wr;
	int i;

	for (i = 0; i < n; ++i)
		if (ibv_post_recv(ctx->qp, &wr, &bad_wr))
			break;

	return i;
}



static int pp_post_send(struct pingpong_context *ctx, int server_count)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size,
		// .length = 32,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_send_wr wr = {
		.wr_id	    = PINGPONG_SEND_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
		.opcode     = IBV_WR_SEND,
		.send_flags = ctx->send_flags,
	};
	struct ibv_send_wr *bad_wr;
	for (int i = 0; i < server_count; ++i) {
	
	 if(ibv_post_send(ctx->qp[i], &wr, &bad_wr))
	 	{
			return 1;
	 	}
	}
	return 0;
}

static int client_pp_post_send_push(struct pingpong_context *ctx, int i)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		.length = ctx->size,
		// .length = requiresize,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_send_wr wr = {
		.wr_id	    = PINGPONG_SEND_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
		.opcode     = IBV_WR_SEND,
		.send_flags = ctx->send_flags,
	};
	struct ibv_send_wr *bad_wr;

	return ibv_post_send(ctx->qp[i], &wr, &bad_wr);
}

static int client_pp_post_send(struct pingpong_context *ctx, int i)
{
	struct ibv_sge list = {
		.addr	= (uintptr_t) ctx->buf,
		// .length = ctx->size,
		.length = requiresize,
		.lkey	= ctx->mr->lkey
	};
	struct ibv_send_wr wr = {
		.wr_id	    = PINGPONG_SEND_WRID,
		.sg_list    = &list,
		.num_sge    = 1,
		.opcode     = IBV_WR_SEND,
		.send_flags = ctx->send_flags,
	};
	struct ibv_send_wr *bad_wr;

	return ibv_post_send(ctx->qp[i], &wr, &bad_wr);
}



struct ts_params {
	uint64_t		 comp_recv_max_time_delta;
	uint64_t		 comp_recv_min_time_delta;
	uint64_t		 comp_recv_total_time_delta;
	uint64_t		 comp_recv_prev_time;
	int			 last_comp_with_ts;
	unsigned int		 comp_with_time_iters;
};


// static inline int reserve_parse_single_wc(struct pingpong_context *ctx, int *scnt,
// 				  int *rcnt, int *routs, int iters,
// 				  uint64_t wr_id, enum ibv_wc_status status,
// 				  uint64_t completion_timestamp,
// 				  struct ts_params *ts)
// {
// 	if (status != IBV_WC_SUCCESS) {
// 		fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
// 			ibv_wc_status_str(status),
// 			status, (int)wr_id);
// 		return 1;
// 	}

// 	switch ((int)wr_id) {
// 	case PINGPONG_SEND_WRID:
// 		++(*scnt);
// 		break;

// 	case PINGPONG_RECV_WRID:
// 		if (--(*routs) <= 1) {
// 			*routs += reserve_pp_post_recv(ctx, ctx->rx_depth - *routs);
// 			if (*routs < ctx->rx_depth) {
// 				fprintf(stderr,
// 					"Couldn't post receive (%d)\n",
// 					*routs);
// 				return 1;
// 			}
// 		}

// 		++(*rcnt);
// 		if (use_ts) {
// 			if (ts->last_comp_with_ts) {
// 				uint64_t delta;

// 				/* checking whether the clock was wrapped around */
// 				if (completion_timestamp >= ts->comp_recv_prev_time)
// 					delta = completion_timestamp - ts->comp_recv_prev_time;
// 				else
// 					delta = ctx->completion_timestamp_mask - ts->comp_recv_prev_time +
// 						completion_timestamp + 1;

// 				ts->comp_recv_max_time_delta = max(ts->comp_recv_max_time_delta, delta);
// 				ts->comp_recv_min_time_delta = min(ts->comp_recv_min_time_delta, delta);
// 				ts->comp_recv_total_time_delta += delta;
// 				ts->comp_with_time_iters++;
// 			}

// 			ts->comp_recv_prev_time = completion_timestamp;
// 			ts->last_comp_with_ts = 1;
// 		} else {
// 			ts->last_comp_with_ts = 0;
// 		}

// 		break;

// 	default:
// 		fprintf(stderr, "Completion for unknown wr_id %d\n",
// 			(int)wr_id);
// 		return 1;
// 	}

// 	ctx->pending &= ~(int)wr_id;
// 	if (*scnt < iters && !ctx->pending) {
// 		if (pp_post_send(ctx)) {
// 			fprintf(stderr, "Couldn't post send\n");
// 			return 1;
// 		}
// 		ctx->pending = PINGPONG_RECV_WRID |
// 			PINGPONG_SEND_WRID;
// 	}

// 	return 0;
// }


// static inline int parse_single_wc(struct pingpong_context *ctx, int *scnt,
// 				  int *rcnt, int *routs, int iters, int server_count,
// 				  uint64_t wr_id, enum ibv_wc_status status,
// 				  uint64_t completion_timestamp,
// 				  struct ts_params *ts)
// {
//     //routs 与 scnt，rcnt都要设置为单独的
// 	if (status != IBV_WC_SUCCESS) {
// 		fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
// 			ibv_wc_status_str(status),
// 			status, (int)wr_id);
// 		return 1;
// 	}

// 	switch ((int)wr_id) {
// 	case PINGPONG_SEND_WRID:
// 		++(*scnt);
// 		break;

// 	case PINGPONG_RECV_WRID:
// 		if (--(*routs) <= 1) {
// 			*routs += pp_post_recv(ctx, ctx->rx_depth - *routs);
// 			if (*routs < ctx->rx_depth) {
// 				fprintf(stderr,
// 					"Couldn't post receive (%d)\n",
// 					*routs);
// 				return 1;
// 			}
// 		}

// 		++(*rcnt);

// 		ts->last_comp_with_ts = 0;


// 		break;

// 	default:
// 		fprintf(stderr, "Completion for unknown wr_id %d\n",
// 			(int)wr_id);
// 		return 1;
// 	}

// 	ctx->pending &= ~(int)wr_id;
// 	if (*scnt < iters && !ctx->pending) {
// 		if (pp_post_send(ctx, server_count)) {
// 			fprintf(stderr, "Couldn't post send\n");
// 			return 1;
// 		}
// 		ctx->pending = PINGPONG_RECV_WRID |
// 			PINGPONG_SEND_WRID;
// 	}

// 	return 0;
// }

static inline int client_parse_single_send_wc(struct pingpong_context *ctx, int *scnt,
				  int *rcnt, int *routs, int iters, int server_count,
				  uint64_t wr_id, enum ibv_wc_status status,
				  uint64_t completion_timestamp,
				  struct ts_params *ts)
{
	if (status != IBV_WC_SUCCESS) {
		fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
			ibv_wc_status_str(status),
			status, (int)wr_id);
		return 1;
	}
	switch ((int)wr_id) {
	case PINGPONG_SEND_WRID:
		++(*scnt);
		break;
	default:
		fprintf(stderr, "Completion for unknown wr_id %d\n",
			(int)wr_id);
		return 1;
	}

	// ctx->pending &= ~(int)wr_id;
	if (*scnt < iters) {
		for (int i = 0; i < server_count; ++i) {
        	if (client_pp_post_send_push(ctx, i)) {
            fprintf(stderr, "Couldn't post send\n");
            return 1;
        }
    }
	// 	ctx->pending = PINGPONG_RECV_WRID |
	// 		PINGPONG_SEND_WRID;
	}

	return 0;
}



static inline int client_parse_single_wc(struct pingpong_context *ctx, int *scnt,
				  int *rcnt, int *routs, int iters, int server_count,
				  uint64_t wr_id, enum ibv_wc_status status,
				  uint64_t completion_timestamp,
				  struct ts_params *ts)
{
	if (status != IBV_WC_SUCCESS) {
		fprintf(stderr, "Failed status %s (%d) for wr_id %d\n",
			ibv_wc_status_str(status),
			status, (int)wr_id);
		return 1;
	}
	switch ((int)wr_id) {
	case PINGPONG_SEND_WRID:
		++(*scnt);
		break;
	case PINGPONG_RECV_WRID:
		if (--(*routs) <= 1) {
			*routs += client_pp_post_recv(ctx, ctx->rx_depth - *routs);
			if (*routs < ctx->rx_depth) {
				fprintf(stderr,
					"Couldn't post receive (%d)\n",
					*routs);
				return 1;
			}
		}

		++(*rcnt);
		ts->last_comp_with_ts = 0;
		break;

	default:
		fprintf(stderr, "Completion for unknown wr_id %d\n",
			(int)wr_id);
		return 1;
	}

	ctx->pending &= ~(int)wr_id;
	if (*scnt < iters && !ctx->pending) {
		for (int i = 0; i < server_count; ++i) {
        	if (client_pp_post_send(ctx, i)) {
            fprintf(stderr, "Couldn't post send\n");
            return 1;
        }
    }
		ctx->pending = PINGPONG_RECV_WRID |
			PINGPONG_SEND_WRID;
	}

	return 0;
}



static void usage(const char *argv0)
{
	printf("Usage:\n");
	printf("  %s            start a server and wait for connection\n", argv0);
	printf("  %s <host>     connect to server at <host>\n", argv0);
	printf("\n");
	printf("Options:\n");
	printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
	printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
	printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
	printf("  -s, --size=<size>      size of message to exchange (default 4096)\n");
	printf("  -m, --mtu=<size>       path MTU (default 1024)\n");
	printf("  -r, --rx-depth=<dep>   number of receives to post at a time (default 500)\n");
	printf("  -n, --iters=<iters>    number of exchanges (default 1000)\n");
	printf("  -l, --sl=<sl>          service level value\n");
	printf("  -e, --events           sleep on CQ events (default poll)\n");
	printf("  -g, --gid-idx=<gid index> local port gid index\n");
	printf("  -o, --odp		    use on demand paging\n");
	printf("  -t, --ts	            get CQE with timestamp\n");
	printf("  --startsize=<size>     start size of message to exchange (default 1024)\n");
    printf("  --endsize=<size>       end size of message to exchange (maximum size to test)\n");
	printf("  --reqsize=<size>       requirements of message to exchange(default 32) \n");
	printf("  -c, --chk	            validate received buffer\n");
    printf("  -M, --mode=<mode>       operation mode: 'push' or 'pull' (default 'push')\n");
}

int main(int argc, char *argv[])
{
	struct ibv_device      **dev_list;
	struct ibv_device	*ib_dev;
	struct pingpong_context *ctx;
	struct pingpong_dest     my_dest;
	struct pingpong_dest    *rem_dest;
	// struct timeval           start, end;
	struct timespec start, end;
	char                    *ib_devname = NULL;
	char                    *servername = NULL;
	int 					server_count = 0;
	char 					*server_addresses[10];
	unsigned int             port = 18515;
	int                      ib_port = 1;
	unsigned int             size = 4096;
	unsigned int startsize = 1024; // 默认起始大小为1024字节
    unsigned int endsize = 64 * 1024 * 1024;   // 默认结束大小为4096字节
	unsigned int reqsize = 32;
    char *mode = "pull"; // 默认为pull
	enum ibv_mtu		 mtu = IBV_MTU_1024;
	unsigned int             rx_depth = 500;
	unsigned int             iters = 1000;
	int                      use_event = 0;
	int                      routs;
	int                      rcnt, scnt;
	int                      num_cq_events = 0;
	int                      sl = 0;
	int			 gidx = -1;
	char			 gid[33];
	struct ts_params	 ts;
	double result[MAX_COMMUNICATION_ITERS];

	srand48(getpid() * time(NULL));

	while (1) {
		int c;

		static struct option long_options[] = {
			{ .name = "port",     .has_arg = 1, .val = 'p' },
			{ .name = "ib-dev",   .has_arg = 1, .val = 'd' },
			{ .name = "ib-port",  .has_arg = 1, .val = 'i' },
			{ .name = "size",     .has_arg = 1, .val = 's' },
			{ .name = "mtu",      .has_arg = 1, .val = 'm' },
			{ .name = "rx-depth", .has_arg = 1, .val = 'r' },
			{ .name = "iters",    .has_arg = 1, .val = 'n' },
			{ .name = "sl",       .has_arg = 1, .val = 'l' },
			{ .name = "events",   .has_arg = 0, .val = 'e' },
			{ .name = "gid-idx",  .has_arg = 1, .val = 'g' },
			{ .name = "odp",      .has_arg = 0, .val = 'o' },
			{ .name = "ts",       .has_arg = 0, .val = 't' },
			{ .name = "chk",      .has_arg = 0, .val = 'c' },
            { .name = "mode", .has_arg = 1, .val = 'M' },
			{ .name = "startsize", .has_arg = 1, .val = 'a' }, // 新选项
            { .name = "endsize",   .has_arg = 1, .val = 'b' }, // 新选项
            { .name = "reqsize",   .has_arg = 1, .val = 'q' }, // 新选项
			{}
		};

		c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:otcM:",
				long_options, NULL);

		if (c == -1)
			break;

		switch (c) {
		case 'p':
			port = strtoul(optarg, NULL, 0);
			if (port > 65535) {
				usage(argv[0]);
				return 1;
			}
			break;

		case 'd':
			ib_devname = strdupa(optarg);
			break;

		case 'i':
			ib_port = strtol(optarg, NULL, 0);
			if (ib_port < 1) {
				usage(argv[0]);
				return 1;
			}
			break;

		case 's':
			size = strtoul(optarg, NULL, 0);
			break;

		case 'm':
			mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
			if (mtu == 0) {
				usage(argv[0]);
				return 1;
			}
			break;
        case 'M':
            mode = strdup(optarg);
            if (strcmp(mode, "push") != 0 && strcmp(mode, "pull") != 0) {
                fprintf(stderr, "Invalid mode. Use 'push' or 'pull'.\n");
                return 1;
            }
            break;
		case 'r':
			rx_depth = strtoul(optarg, NULL, 0);
			break;

		case 'n':
			iters = strtoul(optarg, NULL, 0);
			break;

		case 'l':
			sl = strtol(optarg, NULL, 0);
			break;

		case 'e':
			++use_event;
			break;

		case 'g':
			gidx = strtol(optarg, NULL, 0);
			break;

		case 'o':
			use_odp = 1;
			break;
		case 't':
			use_ts = 1;
			break;
		case 'a':
			startsize = strtoul(optarg, NULL, 0);
			break;
		case 'b':
			endsize = strtoul(optarg, NULL , 0);
			break;
		case 'q':
			reqsize = strtoul(optarg, NULL, 0);
			break;
		case 'c':
			validate_buf = 1;
			break;

		default:
			usage(argv[0]);
			return 1;
		}
	}

	if (optind == argc - 1){
		servername = strdupa(argv[optind]);
		char *token = strtok(servername, ",");
		while (token != NULL) {
        	server_addresses[server_count++] = strdupa(token);
        	token = strtok(NULL, ",");
    	}
		for (int i = 0; i < server_count; ++i) {
			// printf("Server %d: %s\n", i, server_addresses[i]);
			// 在这里为每个服务器地址建立连接...
		}
	}
	else if (optind < argc) {
		usage(argv[0]);
		return 1;
	}
	

	
	page_size = sysconf(_SC_PAGESIZE);
	for (unsigned int current_size = startsize; current_size <= endsize; current_size *= 2) {
	
    
    	// printf("Testing size: %u bytes\n", current_size);
		size = current_size;
		sleep(2);
	// printf(">>>1\n");
	dev_list = ibv_get_device_list(NULL);
	if (!dev_list) {
		perror("Failed to get IB devices list");
		return 1;
	}

	if (!ib_devname) {
		ib_dev = *dev_list;
		if (!ib_dev) {
			fprintf(stderr, "No IB devices found\n");
			return 1;
		}
	} else {
		int i;
		for (i = 0; dev_list[i]; ++i)
			if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
				break;
		ib_dev = dev_list[i];
		if (!ib_dev) {
			fprintf(stderr, "IB device %s not found\n", ib_devname);
			return 1;
		}
	}
	
	ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, server_count, use_event, server_count);
	if (!ctx)
		return 1;
	
    for (int i = 0; i < ctx->num_servers; ++i) {
        routs = pp_post_recv_many(ctx, ctx->rx_depth,i);
        if (routs < ctx->rx_depth) {
            fprintf(stderr, "Couldn't post receive (%d)\n", routs);
            return 1;
        }
    }

	if (use_event)

		for (int i = 0; i < server_count; ++i) {
			if (ibv_req_notify_cq(ctx->cq[i], 0)) {
				fprintf(stderr, "Couldn't request CQ notification\n");
				return 1;
			}
		}
	
	if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) {
		fprintf(stderr, "Couldn't get port info\n");
		return 1;
	}

	my_dest.lid = ctx->portinfo.lid;
	if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET &&
							!my_dest.lid) {
		fprintf(stderr, "Couldn't get local LID\n");
		return 1;
	}

	if (gidx >= 0) {
		if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) {
			fprintf(stderr, "can't read sgid of index %d\n", gidx);
			return 1;
		}
	} else
		memset(&my_dest.gid, 0, sizeof my_dest.gid);

	struct pingpong_dest *my_dests = calloc(ctx->num_servers, sizeof(struct pingpong_dest));
    // my_dest.qpn = ctx->qp->qp_num;
    // my_dest.psn = lrand48() & 0xffffff;
	for (int i = 0; i < ctx->num_servers; ++i) {
        my_dests[i].lid = ctx->portinfo.lid;
        if (gidx >= 0) {
            if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dests[i].gid)) {
                fprintf(stderr, "Can't read GID of index %d for server %d\n", gidx, i);
                // 错误处理
            }
        } else {
            memset(&my_dests[i].gid, 0, sizeof(my_dests[i].gid));
        }

		my_dests[i].qpn = ctx->qp[i]->qp_num; // 注意：每个qp[i]对应不同的服务器
		my_dests[i].psn = lrand48() & 0xffffff;
		// 对gid等其他属性也执行相应的设置
        char gid[INET6_ADDRSTRLEN];
        inet_ntop(AF_INET6, &my_dests[i].gid, gid, sizeof gid);
        // printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
        //     my_dests[i].lid, my_dests[i].qpn, my_dests[i].psn, gid);

        // printf("  Local address for server %d: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
        //    i, ctx->portinfo.lid, my_dests[i].qpn, my_dests[i].psn, gid);
	}
    
	
	// printf("  local address:  LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
	//        my_dests[i].lid, my_dests[i].qpn, my_dests[i].psn, gid);



	// printf("client");
	// rem_dest = pp_client_exch_dest(servername, port, &my_dest);
    
    struct pingpong_dest *rem_dests = calloc(server_count, sizeof(struct pingpong_dest));
    for (int i = 0; i < server_count; ++i) {
        // printf("Connecting to server %s\n", server_addresses[i]);
        
        if (!pp_client_exch_dest(server_addresses[i], port, &my_dests[i] , &rem_dests[i])) {
            fprintf(stderr, "Failed to exchange destination information with server %s\n", server_addresses[i]);
            // 错误处理，比如清理已分配资源等
            free(rem_dests);
            return 1; // 或其他适当的错误码
        }
        inet_ntop(AF_INET6, &rem_dests[i].gid, gid, sizeof gid);
	    // printf("  remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n",
	    //    rem_dests[i].lid, rem_dests[i].qpn, rem_dests[i].psn, gid);
        // printf("1234\n");


    }



    // printf("1234\n");
	for (int i = 0; i < server_count; ++i) {
		if (pp_connect_ctx(ctx, ib_port, my_dests[i].psn, mtu, sl, &rem_dests[i],
					gidx, i))
			return 1;
    }
    // printf("1234\n");
	ctx->pending = PINGPONG_RECV_WRID;
    // printf("12345\n");


    if (validate_buf)
        for (int i = 0; i < size; i += page_size)
            ctx->buf[i] = i / page_size % sizeof(char);
    // printf("1234567\n");
    
    for (int i = 0; i < server_count; ++i) {
		if (strcmp(mode,"pull") == 0){
        if (client_pp_post_send(ctx, i)) {
            fprintf(stderr, "Couldn't post send\n");
            return 1;
        }
		}
		else if(strcmp(mode,"push") == 0){
        if (client_pp_post_send_push(ctx, i)) {
            fprintf(stderr, "Couldn't post send\n");
            return 1;
        }
		}
    }
    ctx->pending |= PINGPONG_SEND_WRID;

    // printf("123456\n");
	// if (gettimeofday(&start, NULL)) {
	// 	perror("gettimeofday");
	// 	return 1;
	// }


// 需要解决
	
	if (strcmp(mode, "pull") == 0) {
	int idx = -1;
	rcnt = scnt = 0;
	while (rcnt < iters && scnt < iters) {
		clock_gettime(CLOCK_MONOTONIC, &start);
		// printf("RCNT:%d========SCNT:%d\n", rcnt, scnt);
		int ret;
		int ne = 0, i;
		struct ibv_wc wc[100];
		// printf("Big in\n");
        do {
			for (i = 0; i < server_count; ++i) {
				ne = ibv_poll_cq(ctx->cq[i], 2, wc);
				if (ne < 0) {
					fprintf(stderr, "poll CQ failed %d\n", ne);
					return 1;
				}
			}
            
        } while (!use_event && ne < 1);
		// printf("Big out\n");
		
        for (i = 0; i < ne; ++i) {
			// printf("in\n");
            ret = client_parse_single_wc(ctx, &scnt, &rcnt, &routs,
                            iters,
							server_count,
                            wc[i].wr_id,
                            wc[i].status,
                            0, &ts);
			// printf("out\n");
            if (ret) {
                fprintf(stderr, "parse WC failed %d\n", ne);
                return 1;
            }
        }
		clock_gettime(CLOCK_MONOTONIC, &end);
		long seconds, nanoseconds;
		double microseconds;
		seconds = end.tv_sec - start.tv_sec;
		nanoseconds = end.tv_nsec - start.tv_nsec;
		if (nanoseconds < 0) {
			--seconds;
			nanoseconds += 1000000000;
		}
		microseconds = seconds * 1000000 + nanoseconds / 1000.0;
		// unsigned long long elapsed_us = (end.tv_sec - start.tv_sec) * 1000000ULL + (end.tv_usec - start.tv_usec);
        
        // if (end.tv_usec < start.tv_usec){
        // // printf("%llu\n,%llu\n",end.tv_sec,start.tv_sec);
        // }else {
        // double elapsed_us = (double)(end.tv_sec - start.tv_sec) * 1000000.0 + (double)(end.tv_usec - start.tv_usec);

        // long long bytes = (long long) size * 2 * server_count;
        // result[++idx] = elapsed_us;
        result[++idx] = microseconds;
        // }
		// printf("%lld bytes in %.2f microseconds = %.2f Mbit/sec\n",
		//        bytes, (double)elapsed_us, bytes * 8. / (elapsed_us));
		// printf("%d iters in %.2f microseconds = %.2f usec/iter\n",
		//        iters, (double)elapsed_us , ((double)elapsed_us / iters));
	}
	merge_sort(result, 0, idx);
	double sum = 0;
	for (int i = 0; i <= idx; i ++) sum += result[i];
	printf("Valid Count: %d\n", idx);
	double avg = sum / (idx + 1);
	// printf("Time min:%lldmillion second\nTime 50\%:%lldmillion second\nTime 99\%:%lldmillion second\n",result[0], result[(idx + 1) / 2], result[(int)((idx + 1) * 0.99)]);
	printf("%lld bytes  %d iters min:%.3fus\t  50\%:%.3fus\t 99\%:%.3fus\t 999\%:%.3fus\t Max:%.3fus\tAvg:%.3fus\n",
    size,iters,result[0],result[(idx + 1) / 2], result[(int)((idx + 1) * 0.99)], 
    result[(int)((idx + 1) * 0.999)],result[idx], avg);
    // for (i = 0;i<= idx;i++){
    //     result[i] = 0;
    // }

	
		// float usec = (end.tv_sec - start.tv_sec) * 1000000 +
		// 	(end.tv_usec - start.tv_usec);
		// double seconds = (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec) / 1E9;
		// long long elapsed_us = (end.tv_sec - start.tv_sec) * 1000000LL + (end.tv_usec - start.tv_usec) / 1000;
		// long long bytes = (long long) size * iters * 2;

		// // printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
		// //        bytes, usec / 1000000., bytes * 8. / usec);
		// // printf("%d iters in %.2f seconds = %.2f usec/iter\n",
		// //        iters, usec / 1000000., usec / iters);
		// printf("%lld bytes in %.2f microseconds = %.2f Mbit/sec\n",
		//        bytes, (double)elapsed_us, bytes * 8. / (elapsed_us));
		// printf("%d iters in %.2f microseconds = %.2f usec/iter\n",
		//        iters, (double)elapsed_us , ((double)elapsed_us / iters));
		// if (ts.comp_with_time_iters) {
		// 	printf("Max receive completion clock cycles = %" PRIu64 "\n",
		// 	       ts.comp_recv_max_time_delta);
		// 	printf("Min receive completion clock cycles = %" PRIu64 "\n",
		// 	       ts.comp_recv_min_time_delta);
		// 	printf("Average receive completion clock cycles = %f\n",
		// 	       (double)ts.comp_recv_total_time_delta / ts.comp_with_time_iters);
		// }
		// printf("10\n");	
		if ((!servername) && (validate_buf)) {
			for (int i = 0; i < size; i += page_size)
				if (ctx->buf[i] != i / page_size % sizeof(char))
					printf("invalid data in page %d\n",
					       i / page_size);
		}
		// printf("11\n");	
	
	for (int i = 0; i < server_count; ++i) {
		ibv_ack_cq_events(ctx->cq[i], num_cq_events);
	}
	// printf("12\n");	
	if (pp_close_ctx(ctx, server_count))
		return 1;
	// printf("13\n");	
	ibv_free_device_list(dev_list);
	// printf("15\n");
	free(rem_dests);
	// printf("16\n");
	
	// printf("14\n");
	// printf("end\n");
	// if (gettimeofday(&end, NULL)) {
	// 	perror("gettimeofday");
	// 	return 1;
	// }
	// sleep(2);
	// printf("12345678");
// 需要解决
	}
	else if (strcmp(mode, "push") == 0){
	int idx = -1;
	scnt = 0;
	while (scnt < iters) {
		clock_gettime(CLOCK_MONOTONIC, &start);
		printf("RCNT:%d========SCNT:%d\n", rcnt, scnt);
		int ret;
		int ne = 0, i;
		struct ibv_wc wc[100];
		// printf("Big in\n");
        do {
			for (i = 0; i < server_count; ++i) {
				ne = ibv_poll_cq(ctx->cq[i], 2, wc);
				if (ne < 0) {
					fprintf(stderr, "poll CQ failed %d\n", ne);
					return 1;
				}
			}
            
        } while (!use_event && ne < 1);
		// printf("Big out\n");
		
        for (i = 0; i < ne; ++i) {
			// printf("in\n");
            ret = client_parse_single_send_wc(ctx, &scnt, &rcnt, &routs,
                            iters,
							server_count,
                            wc[i].wr_id,
                            wc[i].status,
                            0, &ts);
			// printf("out\n");
            if (ret) {
                fprintf(stderr, "parse WC failed %d\n", ne);
                return 1;
            }
        }
		clock_gettime(CLOCK_MONOTONIC, &end);
		long seconds, nanoseconds;
		double microseconds;
		seconds = end.tv_sec - start.tv_sec;
		nanoseconds = end.tv_nsec - start.tv_nsec;
		if (nanoseconds < 0) {
			--seconds;
			nanoseconds += 1000000000;
		}
		microseconds = seconds * 1000000 + nanoseconds / 1000.0;
		// unsigned long long elapsed_us = (end.tv_sec - start.tv_sec) * 1000000ULL + (end.tv_usec - start.tv_usec);
        
        // if (end.tv_usec < start.tv_usec){
        // // printf("%llu\n,%llu\n",end.tv_sec,start.tv_sec);
        // }else {
        // double elapsed_us = (double)(end.tv_sec - start.tv_sec) * 1000000.0 + (double)(end.tv_usec - start.tv_usec);

        // long long bytes = (long long) size * 2 * server_count;
        // result[++idx] = elapsed_us;
        result[++idx] = microseconds;
        // }
		// printf("%lld bytes in %.2f microseconds = %.2f Mbit/sec\n",
		//        bytes, (double)elapsed_us, bytes * 8. / (elapsed_us));
		// printf("%d iters in %.2f microseconds = %.2f usec/iter\n",
		//        iters, (double)elapsed_us , ((double)elapsed_us / iters));
	}
	merge_sort(result, 0, idx);
	double sum = 0;
	for (int i = 0; i <= idx; i ++) sum += result[i];
	printf("Valid Count: %d\n", idx);
	double avg = sum / (idx + 1);
	// printf("Time min:%lldmillion second\nTime 50\%:%lldmillion second\nTime 99\%:%lldmillion second\n",result[0], result[(idx + 1) / 2], result[(int)((idx + 1) * 0.99)]);
	printf("%lld bytes  %d iters min:%.3fus\t  50\%:%.3fus\t 99\%:%.3fus\t 999\%:%.3fus\t Max:%.3fus\tAvg:%.3fus\n",
    size,iters,result[0],result[(idx + 1) / 2], result[(int)((idx + 1) * 0.99)], 
    result[(int)((idx + 1) * 0.999)],result[idx], avg);
    // for (i = 0;i<= idx;i++){
    //     result[i] = 0;
    // }

	
		// float usec = (end.tv_sec - start.tv_sec) * 1000000 +
		// 	(end.tv_usec - start.tv_usec);
		// double seconds = (end.tv_sec - start.tv_sec) + (end.tv_usec - start.tv_usec) / 1E9;
		// long long elapsed_us = (end.tv_sec - start.tv_sec) * 1000000LL + (end.tv_usec - start.tv_usec) / 1000;
		// long long bytes = (long long) size * iters * 2;

		// // printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n",
		// //        bytes, usec / 1000000., bytes * 8. / usec);
		// // printf("%d iters in %.2f seconds = %.2f usec/iter\n",
		// //        iters, usec / 1000000., usec / iters);
		// printf("%lld bytes in %.2f microseconds = %.2f Mbit/sec\n",
		//        bytes, (double)elapsed_us, bytes * 8. / (elapsed_us));
		// printf("%d iters in %.2f microseconds = %.2f usec/iter\n",
		//        iters, (double)elapsed_us , ((double)elapsed_us / iters));
		// if (ts.comp_with_time_iters) {
		// 	printf("Max receive completion clock cycles = %" PRIu64 "\n",
		// 	       ts.comp_recv_max_time_delta);
		// 	printf("Min receive completion clock cycles = %" PRIu64 "\n",
		// 	       ts.comp_recv_min_time_delta);
		// 	printf("Average receive completion clock cycles = %f\n",
		// 	       (double)ts.comp_recv_total_time_delta / ts.comp_with_time_iters);
		// }
		// printf("10\n");	
		if ((!servername) && (validate_buf)) {
			for (int i = 0; i < size; i += page_size)
				if (ctx->buf[i] != i / page_size % sizeof(char))
					printf("invalid data in page %d\n",
					       i / page_size);
		}
		// printf("11\n");	
	
	for (int i = 0; i < server_count; ++i) {
		ibv_ack_cq_events(ctx->cq[i], num_cq_events);
	}
	// printf("12\n");	
	if (pp_close_ctx(ctx, server_count))
		return 1;
	// printf("13\n");	
	ibv_free_device_list(dev_list);
	// printf("15\n");
	free(rem_dests);
	// printf("16\n");
	
	// printf("14\n");
	}




    
    }

	return 0;
}